{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f57d3a70",
   "metadata": {},
   "source": [
    "给模型参数增加随机性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91162df8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAUJ0lEQVR4nO3dbUxb970H8K8NtoHAMYUEe9zgG6RVS7l5WEcSctIrdVq9sJY7NStX6raoY1WUqqmJmjJFKlKTKl0lejNddeuakhfbkr7pssuusqkobcQlLVFVJzR0bIQktNW6C01iu3ngGGiwjf27L3o5mxuSYgL+2/H3Ix0p5///Yf/OIf5yfI4fLCIiICJSwKq6ASLKXQwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJSRlkA7du3D8uWLUNBQQHq6urQ29urqhUiUkRJAP3ud79DS0sLnn32Wbz//vtYvXo16uvrEQqFVLRDRIpYVLwZta6uDmvXrsXLL78MAEgkEqiqqsL27dvx9NNPp7sdIlIkP913GI1G0dfXh9bWVnPMarXC6/XC7/fP+DORSASRSMRcTyQSuHLlCsrLy2GxWBa8ZyJKjYhgbGwMlZWVsFpv/EQr7QF06dIlxONxuFyupHGXy4Vz587N+DNtbW3Ys2dPOtojonk0MjKCpUuX3nA+7QE0F62trWhpaTHXDcOAx+PByMgINE1T2BkRzSQcDqOqqgolJSU3rUt7AC1evBh5eXkIBoNJ48FgEG63e8afcTgccDgc141rmsYAIspgX3aKJO1Xwex2O2pra9Hd3W2OJRIJdHd3Q9f1dLdDRAopeQrW0tKCpqYmrFmzBuvWrcPPf/5zTExM4NFHH1XRDhEpoiSAHn74YXz66afYvXs3AoEAvv71r+PNN9+87sQ0Ed3elLwO6FaFw2E4nU4YhsFzQEQZaLaPUb4XjIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMqkHEDHjx/Hd7/7XVRWVsJiseAPf/hD0ryIYPfu3fjKV76CwsJCeL1efPjhh0k1V65cwebNm6FpGkpLS7FlyxaMj4/f0oYQUfZJOYAmJiawevVq7Nu3b8b5vXv34qWXXsL+/ftx8uRJLFq0CPX19ZicnDRrNm/ejMHBQXR1daGzsxPHjx/HY489NvetIKLsJLcAgBw+fNhcTyQS4na75Wc/+5k5Njo6Kg6HQ37729+KiMiZM2cEgLz33ntmzRtvvCEWi0XOnz8/q/s1DEMAiGEYt9I+ES2Q2T5G5/Uc0Mcff4xAIACv12uOOZ1O1NXVwe/3AwD8fj9KS0uxZs0as8br9cJqteLkyZMz3m4kEkE4HE5aiCj7zWsABQIBAIDL5Uoad7lc5lwgEEBFRUXSfH5+PsrKysyaL2pra4PT6TSXqqqq+WybiBTJiqtgra2tMAzDXEZGRlS3RETzYF4DyO12AwCCwWDSeDAYNOfcbjdCoVDS/NTUFK5cuWLWfJHD4YCmaUkLEWW/eQ2g6upquN1udHd3m2PhcBgnT56ErusAAF3XMTo6ir6+PrPm2LFjSCQSqKurm892iCjD5af6A+Pj4/joo4/M9Y8//hj9/f0oKyuDx+PBjh078Pzzz+POO+9EdXU1du3ahcrKSmzatAkAcNddd+E73/kOtm7div379yMWi6G5uRnf//73UVlZOW8bRkRZINXLa2+99ZYAuG5pamoSkc8vxe/atUtcLpc4HA657777ZGhoKOk2Ll++LD/4wQ+kuLhYNE2TRx99VMbGxub9Eh8RqTHbx6hFRERh/s1JOByG0+mEYRg8H0SUgWb7GM2Kq2BEdHtiABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKpPy1PETzRSQBY2QQ8chn5ljRkn9GYenMX1BJtx8GECkj8Tg+OfHfuHb1gjnm2fAwAyiH8CkYKSOJOEQSqtsghRhApIxIHJJgAOUyBhAp8/kRUFx1G6QQA4iUkUSCR0A5jgFEyiSmopD4VNKYJc+mqBtSgQFEysSuhTE1OW6uW/JsKHBWKOyI0i2lAGpra8PatWtRUlKCiooKbNq0CUNDQ0k1k5OT8Pl8KC8vR3FxMRobGxEMBpNqhoeH0dDQgKKiIlRUVGDnzp2Ymkr+S0g5QASAmKsWiwVWHgHllJQCqKenBz6fDydOnEBXVxdisRg2btyIiYkJs+app57C66+/jo6ODvT09ODChQt46KGHzPl4PI6GhgZEo1G8++67ePXVV3Hw4EHs3r17/raKspYljy9NyylyC0KhkACQnp4eEREZHR0Vm80mHR0dZs3Zs2cFgPj9fhEROXLkiFitVgkEAmZNe3u7aJomkUhkVvdrGIYAEMMwbqV9Uix84QPp3b/VXE79ulkmLn+iui2aB7N9jN7SOSDDMAAAZWVlAIC+vj7EYjF4vV6zZvny5fB4PPD7/QAAv9+PlStXwuVymTX19fUIh8MYHByc8X4ikQjC4XDSQrcjC6w8Asopcw6gRCKBHTt24J577sGKFSsAAIFAAHa7HaWlpUm1LpcLgUDArPnH8Jmen56bSVtbG5xOp7lUVVXNtW3KYBaLBRYrAyiXzDmAfD4fTp8+jUOHDs1nPzNqbW2FYRjmMjIysuD3SQtvptcAWay8MJtL5vTnprm5GZ2dnTh+/DiWLl1qjrvdbkSjUYyOjiYdBQWDQbjdbrOmt7c36famr5JN13yRw+GAw+GYS6uUwRLxmOoWSLGU/tyICJqbm3H48GEcO3YM1dXVSfO1tbWw2Wzo7u42x4aGhjA8PAxd1wEAuq5jYGAAoVDIrOnq6oKmaaipqbmVbaEsIwygnJfSEZDP58Nrr72GP/7xjygpKTHP2TidThQWFsLpdGLLli1oaWlBWVkZNE3D9u3boes61q9fDwDYuHEjampq8Mgjj2Dv3r0IBAJ45pln4PP5eJSTY3gERCkFUHt7OwDgm9/8ZtL4gQMH8OMf/xgA8OKLL8JqtaKxsRGRSAT19fV45ZVXzNq8vDx0dnZi27Zt0HUdixYtQlNTE5577rlb2xLKOokpBlCus4iIfHlZZgmHw3A6nTAMA5qmqW6H5ijwl//BiP+/zPX8ghL8y7/vgn1RqbqmaF7M9jHKSw6kzLWr55PWHSXlyLMVKOqGVGAAkTpfOPa25OUDFouaXkgJBhBlDIs1HxYL/0vmEv62KWNY8vJ4BJRjGECUMazWfFgYQDmFAURKzHTx1WLNA/gULKfwt03KfPErefg+sNzD3zipIXLd50ED4FOwHMMAIkUEkuDH8OY6BhApISJIxPmdYLmOAURqCI+AiAFEighmPgdEuYUBRErIVBSRsUtJYwWlM38gHd2+GECkhEgCialo0lh+QYmibkgVBhBlDH4pYe5hAFHGsOTzGzFyDQOIMgaPgHIPA4gyBgMo9zCASIkbvhmVcgoDiJSQ+NSMIUS5hQFESkhiCmAA5TwGECmRiE/hug+FppzDACIlJB7nUzBiAJEafApGQIrfjEo0W/F4HGNjYzecHwsOIxH/+1sxLPkFuDZlgYyOzlhvs9mwaNGi+W6TFGMA0YL44IMP8MADDyAWm/nrl713V+GJhpXm+tWro9j6bw/iU+PajPWbNm3Cyy+/vCC9kjoMIFoQsVgM58+fv2EAXV1WgkC0GiOTd6EoLwxn9DjOX7iAT0c/m7n+6tWFbJcUSekcUHt7O1atWgVN06BpGnRdxxtvvGHOT05Owufzoby8HMXFxWhsbEQwGEy6jeHhYTQ0NKCoqAgVFRXYuXMnpqb4uTC55nLsnzAwfi8ux5ZiZPIu/GXsXxHjf4Ock1IALV26FC+88AL6+vpw6tQpfOtb38KDDz6IwcFBAMBTTz2F119/HR0dHejp6cGFCxfw0EMPmT8fj8fR0NCAaDSKd999F6+++ioOHjyI3bt3z+9WUca7Fi9GXOz/v2ZBOKohxk9ozT1yi+644w751a9+JaOjo2Kz2aSjo8OcO3v2rAAQv98vIiJHjhwRq9UqgUDArGlvbxdN0yQSicz6Pg3DEABiGMattk8L5M9//rPYbDbB5y/2uW7xrq+T/2w7Knue75WfPu+X/3i6TQrsN67/4Q9/qHqTKAWzfYzO+RxQPB5HR0cHJiYmoOs6+vr6EIvF4PV6zZrly5fD4/HA7/dj/fr18Pv9WLlyJVwul1lTX1+Pbdu2YXBwEHfffXdKPZw7dw7FxcVz3QRaQH/9619v+jqf//3kQ5zo/ilCUQ8K88bhiH2E2E2eihuGgTNnzixEq7QAxsfHZ1WXcgANDAxA13VMTk6iuLgYhw8fRk1NDfr7+2G321FaWppU73K5EAgEAACBQCApfKbnp+duJBKJIBKJmOvhcBjA5/8pef4oM93sEjwAfPjJFXz4yTuzvr1oNIrRG1yip8wzMTExq7qUA+hrX/sa+vv7YRgGfv/736OpqQk9PT0pN5iKtrY27Nmz57rxuro6aJq2oPdNc1NcXDyvXzK4ZMkSbNiwYd5ujxbW9EHCl0n5ldB2ux1f/epXUVtbi7a2NqxevRq/+MUv4Ha7Z/wrFQwG4XZ//mHjbrf7uqti0+vTNTNpbW2FYRjmMjIykmrbRJSBbvmtGIlEApFIBLW1tbDZbOju7jbnhoaGMDw8DF3XAQC6rmNgYAChUMis6erqgqZpqKmpueF9OBwO89L/9EJE2S+lp2Ctra24//774fF4MDY2htdeew1vv/02jh49CqfTiS1btqClpQVlZWXQNA3bt2+HrutYv349AGDjxo2oqanBI488gr179yIQCOCZZ56Bz+eDw+FYkA0kosyVUgCFQiH86Ec/wsWLF+F0OrFq1SocPXoU3/72twEAL774IqxWKxobGxGJRFBfX49XXnnF/Pm8vDx0dnZi27Zt0HUdixYtQlNTE5577rn53SpSLi8vD5qm3fCV0KkqLCycl9uhzGKRm10rzVDhcBhOpxOGYfDpWIaKRqPXne+7FUVFRSgvL5+326OFNdvHKN8LRgvCbrejqqpKdRuU4fh5QESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEiZfNUNzIWIAADC4bDiTohoJtOPzenH6o1kZQBdvnwZAFBVVaW4EyK6mbGxMTidzhvOZ2UAlZWVAQCGh4dvunGULBwOo6qqCiMjI9A0TXU7WYH7bG5EBGNjY6isrLxpXVYGkNX6+akrp9PJ/xRzoGka91uKuM9SN5uDA56EJiJlGEBEpExWBpDD4cCzzz4Lh8OhupWswv2WOu6zhWWRL7tORkS0QLLyCIiIbg8MICJShgFERMowgIhImawMoH379mHZsmUoKChAXV0dent7VbekTFtbG9auXYuSkhJUVFRg06ZNGBoaSqqZnJyEz+dDeXk5iouL0djYiGAwmFQzPDyMhoYGFBUVoaKiAjt37sTU1FQ6N0WZF154ARaLBTt27DDHuM/SRLLMoUOHxG63y29+8xsZHByUrVu3SmlpqQSDQdWtKVFfXy8HDhyQ06dPS39/vzzwwAPi8XhkfHzcrHn88celqqpKuru75dSpU7J+/XrZsGGDOT81NSUrVqwQr9crf/rTn+TIkSOyePFiaW1tVbFJadXb2yvLli2TVatWyZNPPmmOc5+lR9YF0Lp168Tn85nr8XhcKisrpa2tTWFXmSMUCgkA6enpERGR0dFRsdls0tHRYdacPXtWAIjf7xcRkSNHjojVapVAIGDWtLe3i6ZpEolE0rsBaTQ2NiZ33nmndHV1yb333msGEPdZ+mTVU7BoNIq+vj54vV5zzGq1wuv1wu/3K+wscxiGAeDvb9jt6+tDLBZL2mfLly+Hx+Mx95nf78fKlSvhcrnMmvr6eoTDYQwODqax+/Ty+XxoaGhI2jcA91k6ZdWbUS9duoR4PJ70SwcAl8uFc+fOKeoqcyQSCezYsQP33HMPVqxYAQAIBAKw2+0oLS1NqnW5XAgEAmbNTPt0eu52dOjQIbz//vt47733rpvjPkufrAogujmfz4fTp0/jnXfeUd1KRhsZGcGTTz6Jrq4uFBQUqG4np2XVU7DFixcjLy/vuqsRwWAQbrdbUVeZobm5GZ2dnXjrrbewdOlSc9ztdiMajWJ0dDSp/h/3mdvtnnGfTs/dbvr6+hAKhfCNb3wD+fn5yM/PR09PD1566SXk5+fD5XJxn6VJVgWQ3W5HbW0turu7zbFEIoHu7m7ouq6wM3VEBM3NzTh8+DCOHTuG6urqpPna2lrYbLakfTY0NITh4WFzn+m6joGBAYRCIbOmq6sLmqahpqYmPRuSRvfddx8GBgbQ399vLmvWrMHmzZvNf3OfpYnqs+CpOnTokDgcDjl48KCcOXNGHnvsMSktLU26GpFLtm3bJk6nU95++225ePGiuXz22WdmzeOPPy4ej0eOHTsmp06dEl3XRdd1c376kvLGjRulv79f3nzzTVmyZElOXVL+x6tgItxn6ZJ1ASQi8stf/lI8Ho/Y7XZZt26dnDhxQnVLygCYcTlw4IBZc+3aNXniiSfkjjvukKKiIvne974nFy9eTLqdv/3tb3L//fdLYWGhLF68WH7yk59ILBZL89ao88UA4j5LD34cBxEpk1XngIjo9sIAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlPk/zpjWBn6eP0EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "        \n",
    "        #没坚持到最后,扣分\n",
    "        if over and self.step_n < 200:\n",
    "            reward = -1000\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "98108dee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 2])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "class Model(torch.nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "        self.fc = torch.nn.Sequential(\n",
    "            torch.nn.Linear(4, 64),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(64, 64),\n",
    "            torch.nn.ReLU(),\n",
    "        )\n",
    "\n",
    "        #输出层参数的均值和标准差\n",
    "        self.weight_mean = torch.nn.Parameter(torch.randn(64, 2))\n",
    "        self.weight_std = torch.nn.Parameter(torch.randn(64, 2))\n",
    "\n",
    "        self.bias_mean = torch.nn.Parameter(torch.randn(2))\n",
    "        self.bias_std = torch.nn.Parameter(torch.randn(2))\n",
    "        \n",
    "    def forward(self, state):\n",
    "        state = self.fc(state)\n",
    "\n",
    "        #正态分布投影,获取输出层的参数\n",
    "        weight = self.weight_mean + torch.randn(64, 2) * self.weight_std\n",
    "        bias = self.bias_mean + torch.randn(2) * self.bias_std\n",
    "\n",
    "        #运行模式下不需要随机性\n",
    "        if not self.training:\n",
    "            weight = self.weight_mean\n",
    "            bias = self.bias_mean\n",
    "\n",
    "        #计算输出\n",
    "        return state.matmul(weight) + bias\n",
    "\n",
    "\n",
    "model = Model()\n",
    "model_delay = Model()\n",
    "\n",
    "#复制参数\n",
    "model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "model(torch.randn(5, 4)).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84cbf0ff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-981.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    data = []\n",
    "    reward_sum = 0\n",
    "\n",
    "    state = env.reset()\n",
    "    over = False\n",
    "    while not over:\n",
    "        #因为模型本身有随机性,所以这里不需要再随机动作\n",
    "        action = model(torch.FloatTensor(state).reshape(1, 4)).argmax().item()\n",
    "\n",
    "        next_state, reward, over = env.step(action)\n",
    "\n",
    "        data.append((state, action, reward, next_state, over))\n",
    "        reward_sum += reward\n",
    "\n",
    "        state = next_state\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    return data, reward_sum\n",
    "\n",
    "\n",
    "play()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7bfcfd21",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_7154/2282743005.py:27: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at  ../torch/csrc/utils/tensor_new.cpp:201.)\n",
      "  state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(202,\n",
       " (array([-0.01686703, -0.03263399, -0.02972687, -0.00819585], dtype=float32),\n",
       "  1,\n",
       "  1.0,\n",
       "  array([-0.0175197 ,  0.1629014 , -0.02989078, -0.31010774], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据池\n",
    "class Pool:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.pool = []\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.pool)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return self.pool[i]\n",
    "\n",
    "    #更新动作池\n",
    "    def update(self):\n",
    "        #每次更新不少于N条新数据\n",
    "        old_len = len(self.pool)\n",
    "        while len(pool) - old_len < 200:\n",
    "            self.pool.extend(play()[0])\n",
    "\n",
    "        #只保留最新的N条数据\n",
    "        self.pool = self.pool[-2_0000:]\n",
    "\n",
    "    #获取一批数据样本\n",
    "    def sample(self):\n",
    "        data = random.sample(self.pool, 64)\n",
    "\n",
    "        state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n",
    "        action = torch.LongTensor([i[1] for i in data]).reshape(-1, 1)\n",
    "        reward = torch.FloatTensor([i[2] for i in data]).reshape(-1, 1)\n",
    "        next_state = torch.FloatTensor([i[3] for i in data]).reshape(-1, 4)\n",
    "        over = torch.LongTensor([i[4] for i in data]).reshape(-1, 1)\n",
    "\n",
    "        return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "pool = Pool()\n",
    "pool.update()\n",
    "pool.sample()\n",
    "\n",
    "len(pool), pool[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "018c4b1e",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 410 -991.5\n",
      "weight= -0.05162937939167023 1.5363906621932983 bias= [-0.14915892481803894, 0.19301483035087585]\n",
      "100 20000 149.75\n",
      "weight= -0.34773126244544983 1.8414790630340576 bias= [-0.3386839032173157, -0.2702609896659851]\n",
      "200 20000 200.0\n",
      "weight= -0.39857396483421326 1.902843713760376 bias= [-0.3145585358142853, -0.17365393042564392]\n",
      "300 20000 200.0\n",
      "weight= -0.37846851348876953 1.895090937614441 bias= [-0.394956111907959, -0.053517378866672516]\n",
      "400 20000 200.0\n",
      "weight= -0.36975133419036865 1.8895775079727173 bias= [-0.39090868830680847, -0.025117414072155952]\n",
      "500 20000 200.0\n",
      "weight= -0.3579482436180115 1.8912266492843628 bias= [-0.39548859000205994, -0.0065963189117610455]\n",
      "600 20000 200.0\n",
      "weight= -0.3552493155002594 1.892190933227539 bias= [-0.38327670097351074, 0.0018533050315454602]\n",
      "700 20000 200.0\n",
      "weight= -0.34987977147102356 1.8944052457809448 bias= [-0.3704076111316681, 1.5890225768089294e-05]\n",
      "800 20000 200.0\n",
      "weight= -0.3472265303134918 1.9024701118469238 bias= [-0.37121301889419556, 0.024293327704072]\n",
      "900 20000 200.0\n",
      "weight= -0.3437800705432892 1.91338312625885 bias= [-0.3766831159591675, 0.02384124882519245]\n"
     ]
    }
   ],
   "source": [
    "#训练\n",
    "def train():\n",
    "    model.train()\n",
    "    model_delay.eval()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=2e-4)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #共更新N轮数据\n",
    "    for epoch in range(1000):\n",
    "        pool.update()\n",
    "\n",
    "        #每次更新数据后,训练N次\n",
    "        for i in range(200):\n",
    "\n",
    "            #采样N条数据\n",
    "            state, action, reward, next_state, over = pool.sample()\n",
    "\n",
    "            #计算value\n",
    "            value = model(state).gather(dim=1, index=action)\n",
    "\n",
    "            #计算target\n",
    "            with torch.no_grad():\n",
    "                target = model_delay(next_state)\n",
    "            target = target.max(dim=1)[0].reshape(-1, 1)\n",
    "            target = target * 0.99 * (1 - over) + reward\n",
    "\n",
    "            loss = loss_fn(value, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "        #复制参数\n",
    "        if (epoch + 1) % 5 == 0:\n",
    "            model_delay.load_state_dict(model.state_dict())\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            model.eval()\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            model.train()\n",
    "            print(epoch, len(pool), test_result)\n",
    "\n",
    "            weight = model.weight_mean + 1 * model.weight_std\n",
    "            bias = model.bias_mean + 1 * model.bias_std\n",
    "            print('weight=',\n",
    "                  weight.mean().item(),\n",
    "                  weight.std().item(), 'bias=', bias.tolist())\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a101d0e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAATH0lEQVR4nO3dfUxb570H8K8N2ISXYwIUewh8w92qpigv3UhCzqqrTYsb1qGqWanU7kYdq6JEyUzUlClXQ2pTJbcSVfZH12wpudLukt4/uuxS3WwqSlshkxL11g0NDRKhCXeVMoHS2M7L9TGQYhv7d//o5axuSIZ58YOT70c6Uvw8Px//zkP8xT4Hg0VEBEREClhVN0BE9y4GEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKaMsgA4fPowVK1YgPz8f9fX16OvrU9UKESmiJID++Mc/orW1FS+99BI++eQTrF27Fg0NDQiFQiraISJFLCo+jFpfX4/169fjt7/9LQAgmUyiuroau3fvxi9/+ctMt0NEiuRm+gFjsRj6+/vR1tZmjlmtVng8Hvj9/hnvE41GEY1GzdvJZBI3btxAWVkZLBbLovdMROkREYyNjaGyshJW6+3faGU8gK5du4ZEIgGn05ky7nQ6cfHixRnv097ejv3792eiPSJaQKOjo6iqqrrtfMYDaC7a2trQ2tpq3jYMA263G6Ojo9A0TWFnRDSTSCSC6upqFBcX37Eu4wFUXl6OnJwcBIPBlPFgMAiXyzXjfex2O+x2+y3jmqYxgIiWsL93iiTjV8FsNhvq6urg8/nMsWQyCZ/PB13XM90OESmk5C1Ya2srmpubsW7dOmzYsAG//vWvMTExgWeffVZFO0SkiJIAeuqpp3D16lXs27cPgUAADz30EN59991bTkwT0d1Nyc8BzVckEoHD4YBhGDwHRLQEzfY5ys+CEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISJm0A+j06dN47LHHUFlZCYvFgj/96U8p8yKCffv24Rvf+AaWLVsGj8eDv/zlLyk1N27cwNatW6FpGkpKSrBt2zaMj4/P60CIKPukHUATExNYu3YtDh8+POP8wYMHcejQIRw5cgRnzpxBYWEhGhoaMDk5adZs3boVQ0ND6O7uRldXF06fPo0dO3bM/SiIKDvJPACQEydOmLeTyaS4XC751a9+ZY6Fw2Gx2+3yhz/8QUREPv30UwEgH3/8sVnzzjvviMVikcuXL8/qcQ3DEABiGMZ82ieiRTLb5+iCngO6dOkSAoEAPB6POeZwOFBfXw+/3w8A8Pv9KCkpwbp168waj8cDq9WKM2fOzLjfaDSKSCSSshFR9lvQAAoEAgAAp9OZMu50Os25QCCAioqKlPnc3FyUlpaaNV/X3t4Oh8NhbtXV1QvZNhEpkhVXwdra2mAYhrmNjo6qbomIFsCCBpDL5QIABIPBlPFgMGjOuVwuhEKhlPmpqSncuHHDrPk6u90OTdNSNiLKfgsaQDU1NXC5XPD5fOZYJBLBmTNnoOs6AEDXdYTDYfT395s1PT09SCaTqK+vX8h2iGiJy033DuPj4/jss8/M25cuXcLAwABKS0vhdruxZ88evPzyy7j//vtRU1ODF198EZWVldiyZQsA4MEHH8QPf/hDbN++HUeOHEE8HkdLSwuefvppVFZWLtiBEVEWSPfy2qlTpwTALVtzc7OIfHkp/sUXXxSn0yl2u102bdokw8PDKfu4fv26/OQnP5GioiLRNE2effZZGRsbW/BLfESkxmyfoxYREYX5NyeRSAQOhwOGYfB8ENESNNvnaFZcBSOiuxMDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlIm7T/LQ7SQEvEojJFBSDLx/yMWaFUPIm9ZsdK+KDMYQKTU1OQ4LvX+B5LxyS8HLBY8+Pi/MIDuEXwLRkTKMICISBkGEBEpwwAiImUYQESkDAOIiJRJK4Da29uxfv16FBcXo6KiAlu2bMHw8HBKzeTkJLxeL8rKylBUVISmpiYEg8GUmpGRETQ2NqKgoAAVFRXYu3cvpqam5n80RJRV0gqg3t5eeL1efPTRR+ju7kY8HsfmzZsxMTFh1jz//PN4++230dnZid7eXnz++ed44oknzPlEIoHGxkbEYjF8+OGHeOONN3Ds2DHs27dv4Y6KiLKDzEMoFBIA0tvbKyIi4XBY8vLypLOz06y5cOGCABC/3y8iIidPnhSr1SqBQMCs6ejoEE3TJBqNzupxDcMQAGIYxnzapyVgMnJNzv77buk7sv3L7d92yFjgM9Vt0TzN9jk6r3NAhmEAAEpLSwEA/f39iMfj8Hg8Zs3KlSvhdrvh9/sBAH6/H6tXr4bT6TRrGhoaEIlEMDQ0NOPjRKNRRCKRlI2Ist+cAyiZTGLPnj14+OGHsWrVKgBAIBCAzWZDSUlJSq3T6UQgEDBrvho+0/PTczNpb2+Hw+Ewt+rq6rm2TURLyJwDyOv14vz58zh+/PhC9jOjtrY2GIZhbqOjo4v+mES0+Ob0YdSWlhZ0dXXh9OnTqKqqMsddLhdisRjC4XDKq6BgMAiXy2XW9PX1pexv+irZdM3X2e122O32ubRKREtYWq+ARAQtLS04ceIEenp6UFNTkzJfV1eHvLw8+Hw+c2x4eBgjIyPQdR0AoOs6BgcHEQqFzJru7m5omoba2tr5HAsRZZm0XgF5vV68+eab+POf/4zi4mLznI3D4cCyZcvgcDiwbds2tLa2orS0FJqmYffu3dB1HRs3bgQAbN68GbW1tXjmmWdw8OBBBAIBvPDCC/B6vXyVQ3SPSSuAOjo6AADf//73U8aPHj2Kn/3sZwCAV199FVarFU1NTYhGo2hoaMDrr79u1ubk5KCrqwu7du2CrusoLCxEc3MzDhw4ML8jIaKsYxERUd1EuiKRCBwOBwzDgKZpqtuheYiOXcf5zv23/EKyIuc31TZG8zLb5yg/C0ZEyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlImrb+MSpQuEcH4+DgSicSM8/GbY8BX/zamAONj45iyh2est1gsKC4uhtXK7513AwYQLapEIoEnn3wSQ0NDM86Xa/l4bec/YZktDwAgksQ/b92K/7kcnrG+oqICPp8Py5cvX6yWKYMYQLSoRAShUAiXL1+ecX7qZiHG4g4MR3UkJBc1+edw9epVXL4cmrE+kUggmUwuZsuUQWm9ju3o6MCaNWugaRo0TYOu63jnnXfM+cnJSXi9XpSVlaGoqAhNTU0IBoMp+xgZGUFjYyMKCgpQUVGBvXv3YmpqamGOhrJOXPIxMLYJgdg/4mr8H3BuzIObCYfqtihD0gqgqqoqvPLKK+jv78fZs2fxgx/8AI8//rj58vr555/H22+/jc7OTvT29uLzzz/HE088Yd4/kUigsbERsVgMH374Id544w0cO3YM+/btW9ijoqyRkFzcTBQDsAAAoskCxCRfbVOUOTJPy5cvl9/97ncSDoclLy9POjs7zbkLFy4IAPH7/SIicvLkSbFarRIIBMyajo4O0TRNotHorB/TMAwBIIZhzLd9WmSxWEweeughATDjdt9yh7z2r0flwMtnZP/LZ+RQ+3/JAyuqb1vvcrnk2rVrqg+L/o7ZPkfnfA4okUigs7MTExMT0HUd/f39iMfj8Hg8Zs3KlSvhdrvh9/uxceNG+P1+rF69Gk6n06xpaGjArl27MDQ0hG9/+9tp9XDx4kUUFRXN9RAoA6ampjA5OXnb+Ztf3MR/nzqEG8lvIik5qLBdwo3/vX7H/Q0PD6OkpGQRuqWFMj4+Pqu6tANocHAQuq5jcnISRUVFOHHiBGprazEwMACbzXbLfwyn04lAIAAACAQCKeEzPT89dzvRaBTRaNS8HYlEAACGYfD80RKXSCRuewkeACYm4/jPnnMAzs1qf8lk0vz609I1MTExq7q0A+iBBx7AwMAADMPAW2+9hebmZvT29qbdYDra29uxf//+W8br6+uhadqiPjbNTzweR2Fh4YLtz2azYf369SgrK1uwfdLCm+03ibR/mstms+Fb3/oW6urq0N7ejrVr1+K1116Dy+VCLBZDOBxOqQ8Gg3C5XAAAl8t1y1Wx6dvTNTNpa2uDYRjmNjo6mm7bRLQEzfvHSZPJJKLRKOrq6pCXlwefz2fODQ8PY2RkBLquAwB0Xcfg4CBCob/9jEd3dzc0TUNtbe1tH8Nut5uX/qc3Isp+ab0Fa2trw6OPPgq3242xsTG8+eabeP/99/Hee+/B4XBg27ZtaG1tRWlpKTRNw+7du6HrOjZu3AgA2Lx5M2pra/HMM8/g4MGDCAQCeOGFF+D1emG32xflAIlo6UorgEKhEH7605/iypUrcDgcWLNmDd577z088sgjAIBXX30VVqsVTU1NiEajaGhowOuvv27ePycnB11dXdi1axd0XUdhYSGam5tx4MCBhT0qWlKKiooW7FVrcXExLBbLguyL1LOIfPWTgNkhEonA4XDAMAy+HVviRATBYBDxeHxB9me1WuFyuZCTk7Mg+6PFMdvnKD8LRovKYrHc8QID3dv4Ow2ISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMrmqG5gLEQEARCIRxZ0Q0Uymn5vTz9XbycoAun79OgCgurpacSdEdCdjY2NwOBy3nc/KACotLQUAjIyM3PHgKFUkEkF1dTVGR0ehaZrqdrIC12xuRARjY2OorKy8Y11WBpDV+uWpK4fDwf8Uc6BpGtctTVyz9M3mxQFPQhORMgwgIlImKwPIbrfjpZdegt1uV91KVuG6pY9rtrgs8veukxERLZKsfAVERHcHBhARKcMAIiJlGEBEpExWBtDhw4exYsUK5Ofno76+Hn19fapbUqa9vR3r169HcXExKioqsGXLFgwPD6fUTE5Owuv1oqysDEVFRWhqakIwGEypGRkZQWNjIwoKClBRUYG9e/diamoqk4eizCuvvAKLxYI9e/aYY1yzDJEsc/z4cbHZbPL73/9ehoaGZPv27VJSUiLBYFB1a0o0NDTI0aNH5fz58zIwMCA/+tGPxO12y/j4uFmzc+dOqa6uFp/PJ2fPnpWNGzfKd7/7XXN+ampKVq1aJR6PR86dOycnT56U8vJyaWtrU3FIGdXX1ycrVqyQNWvWyHPPPWeOc80yI+sCaMOGDeL1es3biURCKisrpb29XWFXS0coFBIA0tvbKyIi4XBY8vLypLOz06y5cOGCABC/3y8iIidPnhSr1SqBQMCs6ejoEE3TJBqNZvYAMmhsbEzuv/9+6e7ulu9973tmAHHNMier3oLFYjH09/fD4/GYY1arFR6PB36/X2FnS4dhGAD+9oHd/v5+xOPxlDVbuXIl3G63uWZ+vx+rV6+G0+k0axoaGhCJRDA0NJTB7jPL6/WisbExZW0ArlkmZdWHUa9du4ZEIpHyRQcAp9OJixcvKupq6Ugmk9izZw8efvhhrFq1CgAQCARgs9lQUlKSUut0OhEIBMyamdZ0eu5udPz4cXzyySf4+OOPb5njmmVOVgUQ3ZnX68X58+fxwQcfqG5lSRsdHcVzzz2H7u5u5Ofnq27nnpZVb8HKy8uRk5Nzy9WIYDAIl8ulqKuloaWlBV1dXTh16hSqqqrMcZfLhVgshnA4nFL/1TVzuVwzrun03N2mv78foVAI3/nOd5Cbm4vc3Fz09vbi0KFDyM3NhdPp5JplSFYFkM1mQ11dHXw+nzmWTCbh8/mg67rCztQREbS0tODEiRPo6elBTU1NynxdXR3y8vJS1mx4eBgjIyPmmum6jsHBQYRCIbOmu7sbmqahtrY2MweSQZs2bcLg4CAGBgbMbd26ddi6dav5b65Zhqg+C56u48ePi91ul2PHjsmnn34qO3bskJKSkpSrEfeSXbt2icPhkPfff1+uXLlibjdv3jRrdu7cKW63W3p6euTs2bOi67roum7OT19S3rx5swwMDMi7774r99133z11SfmrV8FEuGaZknUBJCLym9/8Rtxut9hsNtmwYYN89NFHqltSBsCM29GjR82aL774Qn7+85/L8uXLpaCgQH784x/LlStXUvbz17/+VR599FFZtmyZlJeXyy9+8QuJx+MZPhp1vh5AXLPM4K/jICJlsuocEBHdXRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMv8HiXZmrhmldX8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pt39]",
   "language": "python",
   "name": "conda-env-pt39-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
